package com.chance.cc.crawler.development.scripts.jiuxian;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.KVTag;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.comment;

/**
 * @author lt
 * @version 1.0
 * @date 2021-01-22 15:02:22
 * @email okprog@sina.com
 */
public class JiuXianCrawlerScript extends CrawlerCommonScript {

    private Logger logger = LoggerFactory.getLogger(JiuXianCrawlerScript.class);

    public static final String indexUrlRegex = "https?://www\\.jiuxian\\.com";
    public static final String listUrlRegex = "https://newapplist\\.jiuxian\\.com/search/searchProduct\\S*";
    public static final String productUrlRegex = "https://newappproduct\\.jiuxian\\.com/product/proDetail\\.htm\\S*";
    public static final String commentUrlRegex = "https://newappuser\\.jiuxian\\.com/comment/getProductCommentDetail\\.htm\\S*";

    public static final String listUrlFormat = "https://newapplist.jiuxian.com/search/searchProduct.htm?attrsId=%s:%s&deviceType=ANDROID&orderBy=1:0&pageIndex=%s&pageSize=100";
    public static final String productUrlFormat = "https://newappproduct.jiuxian.com/product/proDetail.htm?deviceType=ANDROID&proId=%s";
    public static final String commentUrlFormat = "https://newappuser.jiuxian.com/comment/getProductCommentDetail.htm?appVersion=8.8.0&deviceType=ANDROID&pager=%s&productId=%s";


    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<>();
        HttpRequest lastRequest = crawlerRecord.getHttpRequest();
        String lastRequestUrl = lastRequest.getUrl();
        int statusCode = page.getStatusCode();
        if (StringUtils.isBlank(page.getRawText()) || (statusCode != 200 && statusCode != 404)){
            crawlerRecord.setNeedWashPage(false);
            crawlerRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
            parsedLinks.add(crawlerRecord);
            return parsedLinks;
        }
        if (lastRequestUrl.matches(indexUrlRegex)){
            String jsons = "{\"jsons\":[{\"id\": 2, \"isAll\": 0, \"list\": [{\"id\": 97, \"isRed\": 0, \"name\": \"\\u9171\\u9999\\u578b\"}, {\"id\": 126, \"isRed\": 1, \"name\": \"\\u6d53\\u9999\\u578b\"}, {\"id\": 132, \"isRed\": 0, \"name\": \"\\u6e05\\u9999\\u578b\"}, {\"id\": 602, \"isRed\": 0, \"name\": \"\\u99a5\\u90c1\\u9999\\u578b\"}, {\"id\": 526, \"isRed\": 0, \"name\": \"\\u51e4\\u9999\\u578b\"}, {\"id\": 1021, \"isRed\": 0, \"name\": \"\\u8463\\u9999\\u578b\"}, {\"id\": 1019, \"isRed\": 0, \"name\": \"\\u8001\\u767d\\u5e72\\u9999\\u578b\"}], \"name\": \"\\u9999\\u578b\"}," +
                    "{\"id\": 5, \"isAll\": 0, \"list\": [{\"id\": 964, \"isRed\": 0, \"name\": \"\\u7ea2\\u8461\\u8404\\u9152\"}, {\"id\": 982, \"isRed\": 0, \"name\": \"\\u767d\\u8461\\u8404\\u9152\"}, {\"id\": 979, \"isRed\": 0, \"name\": \"\\u8d77\\u6ce1\\u9152\"}, {\"id\": 985, \"isRed\": 0, \"name\": \"\\u6843\\u7ea2\\u8461\\u8404\\u9152\"}], \"name\": \"\\u7c7b\\u578b\"}," +
                    "{\"id\": 5, \"isAll\": 0, \"list\": [{\"id\": 6, \"isRed\": 1, \"name\": \"\\u5a01\\u58eb\\u5fcc\"}, {\"id\": 2, \"isRed\": 0, \"name\": \"\\u767d\\u5170\\u5730\"}, {\"id\": 12, \"isRed\": 1, \"name\": \"\\u4f0f\\u7279\\u52a0\"}, {\"id\": 14, \"isRed\": 0, \"name\": \"\\u6717\\u59c6\\u9152\"}, {\"id\": 31, \"isRed\": 0, \"name\": \"\\u9884\\u8c03\\u9152\"}, {\"id\": 17, \"isRed\": 0, \"name\": \"\\u529b\\u5a07\\u9152\"}], \"name\": \"\\u7c7b\\u578b\"}," +
                    "{\"id\": 30, \"isAll\": 1, \"list\": [{\"id\": 117, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/db8569a2ac41450e9984155e956977c6.jpg\", \"isRed\": 0, \"name\": \"\\u9752\\u5c9b\"}, {\"id\": 1330, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/accb66a0a8ae4088ab3102ec409235a4.jpg\", \"isRed\": 0, \"name\": \"\\u74e6\\u4f26\\u4e01\"}, {\"id\": 118, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/a1e5506ce3894182b28e8674a4c53583.jpg\", \"isRed\": 0, \"name\": \"\\u5609\\u58eb\\u4f2f\"}, {\"id\": 122, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/d227dc626b4c44cdba24aa0344453b28.jpg\", \"isRed\": 0, \"name\": \"\\u96ea\\u82b1\"}, {\"id\": 2542, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/9545f77a799e478fa0bb38a3c24ecb44.jpg\", \"isRed\": 0, \"name\": \"\\u57c3\\u4e1d\\u4f2f\\u7235\"}, {\"id\": 2875, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/18672e2c3d1748d2914448313a3ea92e.jpg\", \"isRed\": 0, \"name\": \"\\u8d39\\u5c14\\u5fb7\\u5821\"}], \"name\": \"\\u5564\\u9152\\u54c1\\u724c\"}," +
                    "{\"id\": 10, \"isAll\": 1, \"list\": [{\"id\": 130, \"image\": \"http://img10.jiuxian.com/bill/2017/0322/a69723507f0c4ffc9b29d28be0e48df0.jpg\", \"isRed\": 0, \"name\": \"\\u6930\\u5c9b\"}, {\"id\": 134, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/ebf0e0e92cb243b0a571380989befdbe.jpg\", \"isRed\": 0, \"name\": \"\\u7af9\\u53f6\\u9752\"}, {\"id\": 407, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/9773c9c82feb4922b53917276ee5d5ee.jpg\", \"isRed\": 0, \"name\": \"\\u52b2\\u9152\"}], \"name\": \"\\u517b\\u751f\\u9152\"}," +
                    "{\"id\": 9, \"isAll\": 1, \"list\": [{\"id\": 144, \"image\": \"http://img06.jiuxian.com/bill/2018/0521/16f6b814f6af4f168b37dd2b2c4e5007.jpg\", \"isRed\": 0, \"name\": \"\\u53e4\\u8d8a\\u9f99\\u5c71\"}], \"name\": \"\\u9ec4\\u9152\"}]}";

            JSONObject jsonObject = JSONObject.parseObject(jsons);
            JSONArray jsonArray = jsonObject.getJSONArray("jsons");
            for (Object obj : jsonArray) {
                JSONObject jsonObj = (JSONObject)(obj);
                String id1st = jsonObj.getString("id");
                JSONArray list = jsonObj.getJSONArray("list");
                for (Object o : list) {
                    JSONObject oObj = (JSONObject)(o);
                    String id2nd = oObj.getString("id");

                    String listUrl = String.format(listUrlFormat,id1st,id2nd,1);
                    CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRecord)
                            .httpUrl(listUrl)
                            .recordKey(listUrl)
                            .releaseTime(System.currentTimeMillis())
                            .copyBizTags()
                            .notFilterRecord()
                            .build();
                    parsedLinks.add(requestRecord);
                }
            }
            return parsedLinks;
        }
        if (lastRequestUrl.matches(listUrlRegex)){
            return parseListLinks(crawlerRecord, page, parsedLinks, lastRequestUrl);
        }
        if (lastRequestUrl.matches(productUrlRegex)){
            JSONObject pageObj = JSONObject.parseObject(page.getRawText());
            String productId = pageObj.getJSONObject("result").getString("proId");

            //生成评论record
            CategoryTag categoryTag = crawlerRecord.tagsCreator().scheduleTags().getCategoryTag();
            if (categoryTag.getLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal()) != null){
                String commentUrl = String.format(commentUrlFormat,1,productId);

                CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRecord)
                        .httpUrl(commentUrl)
                        .recordKey(commentUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .notFilterRecord()
                        .needWashed(true)
                        .needParsed(false)
                        .resultLabelTag(comment)
                        .build();
                parsedLinks.add(commentRecord);
            }
            return parsedLinks;
        }
        return null;
    }

    private List<CrawlerRequestRecord> parseListLinks(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks, String lastRequestUrl) {
        JSONObject pageObj = JSONObject.parseObject(page.getRawText());
        JSONObject resultObj = pageObj.getJSONObject("result");
        int totalPageCount = resultObj.getIntValue("pageCount");
        Map<String, Object> urlParams = getUrlParams(lastRequestUrl);
        if (null != urlParams){
            String attrsId = (String) urlParams.get("attrsId");
            String id1st = attrsId.split(":")[0];
            String id2nd = attrsId.split(":")[1];
            int currentPageNum = Integer.parseInt((String)urlParams.get("pageIndex"));
            currentPageNum += 1;
            if (currentPageNum <= totalPageCount){
                String nextPageUrl = String.format(listUrlFormat,id1st,id2nd,currentPageNum);
                CrawlerRequestRecord nextPageRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRecord)
                        .httpUrl(nextPageUrl)
                        .recordKey(nextPageUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .build();
                parsedLinks.add(nextPageRecord);
            }
        }
        JSONArray itemJsonList = resultObj.getJSONArray("resultList");
        for (Object itemJson : itemJsonList) {
            JSONObject itemObj = (JSONObject)itemJson;
            String productIdId = itemObj.getString("proId");
            String productUrl = String.format(productUrlFormat,productIdId);
            CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(productUrl)
                    .recordKey(productUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .needWashed(false)
                    .needParsed(true)
                    .build();
            parsedLinks.add(itemRecord);
        }
        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)){
            crawlerDataList.addAll(washArticle(crawlerRecord, page));
        }
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(comment)){
            crawlerDataList.addAll(washComment(crawlerRecord, page));
        }
        return crawlerDataList;
    }

    private List<CrawlerData> washArticle(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        JSONObject pageObj = JSONObject.parseObject(page.getRawText());
        String productId = pageObj.getJSONObject("result").getString("proId");
        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRecord,page)
                .url(crawlerRecord.getHttpRequest().getUrl())
                .releaseTime(System.currentTimeMillis())
                .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),article.enumVal(),productId))
                .content(page.getRawText())
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                .build();


        crawlerDataList.add(crawlerData);
        return crawlerDataList;
    }

    private List<CrawlerData> washComment(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        JSONObject pageObj = JSONObject.parseObject(page.getRawText());
        JSONArray jsonArray = pageObj.getJSONObject("result").getJSONArray("commentList");
        if(!crawlerRecord.tagsCreator().bizTags().hasKVTag("comment_record_filter_info")){
            logger.error("xcar news crawler comment need to filter information!");
            return crawlerDataList;
        }
        KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
        CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(),CrawlerRecord.class);

        String lastRequestUrl = crawlerRecord.getHttpRequest().getUrl();
        String articleKey = lastRequestUrl.substring(lastRequestUrl.lastIndexOf("=") + 1);
        if (jsonArray.size() > 0){
            for (Object cmt : jsonArray) {
                JSONObject cmtObj = (JSONObject)cmt;
                String commentId = cmtObj.getJSONObject("firstComment").getString("comentId");
                long releaseTime = cmtObj.getLongValue("createTime");
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .url(crawlerRecord.getHttpRequest().getUrl())
                        .releaseTime(releaseTime)
                        .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),comment.enumVal(),commentId))
                        .parentId(StringUtils.joinWith("-",crawlerRecord.getDomain(),article.enumVal(),articleKey))
                        .content(cmtObj.toJSONString())
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .build();
                crawlerData.setFilter(filterInfoRecord.getFilter());
                crawlerData.setFilterInfos(filterInfoRecord.getFilterInfos());
                crawlerData.setFilterKey(crawlerData.getDataId());
                crawlerDataList.add(crawlerData);
            }
        }
        return crawlerDataList;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(indexUrlRegex);
        addUrlRegular(listUrlRegex);
        addUrlRegular(productUrlRegex);
        addUrlRegular(commentUrlRegex);
    }


    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return true;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public String domain() {
        return "jiuxian";
    }

    /**
     * 将url参数转换成map
     * @param url http://*.*.com?aa=11&bb=22&cc=33
     * @return map
     */
    private Map<String, Object> getUrlParams(String url) {
        Map<String, Object> map = new HashMap<String, Object>(0);
        String param = null;
        if (url.contains("?")){
            param = url.split("\\?")[1];
        }
        if (StringUtils.isBlank(param)) {
            return null;
        }
        String[] params = param.split("&");
        for (String s : params) {
            String[] p = s.split("=");
            if (p.length == 2) {
                map.put(p[0], p[1]);
            }
        }
        return map;
    }
}
